{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Notebook for displaying results tables\n",
    "Note: these will only run once you've run `test.py` on the corresponding test sets,\n",
    "e.g. see `scripts/04_eval_checkpoint.sh`, as it pulls the numbers from the respective experiment directories. \n",
    "\n",
    "To run the face forensics tables, you'll need to have processed the face forensics dataset accordingly, following `scripts/00_data_processing_faceforensics_aligned_frames.sh`, and then evalute the corresponding experiments in `scripts/04_eval_checkpoint.sh`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "import glob\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import matplotlib\n",
    "pd.options.display.float_format = '{:,.2f}'.format # print 2 decimal places"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_names_list = {\n",
    "    'gen_models': ['celebahq-pgan-pretrained', 'celebahq-sgan-pretrained',\n",
    "                   'celebahq-glow-pretrained', 'celeba-gmm', 'ffhq-pgan', \n",
    "                   'ffhq-sgan', 'ffhq-sgan2'],\n",
    "    'faceforensics': ['DF', 'NT', 'F2F', 'FS'],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_table(model_paths, model_names, dataset_names, ens_type, partition='val', epoch='epoch_bestval'):\n",
    "    # ens types: voted, avg_before_softmax, avg_after_softmax\n",
    "    metrics_ap = dict()\n",
    "    metrics_acc = dict()\n",
    "    counts = dict()\n",
    "    for model_path, model_name in zip(model_paths, model_names):\n",
    "        metrics_ap[model_name]  = []\n",
    "        metrics_acc[model_name] = []\n",
    "        counts[model_name] = []\n",
    "        for dataset in dataset_names:\n",
    "            metric_file = os.path.join(model_path, partition, epoch, dataset, 'metrics_%s.npz' % ens_type)\n",
    "            if os.path.isfile(metric_file):\n",
    "                metrics = np.load(metric_file)\n",
    "                metrics_ap[model_name].append(metrics['ap'])\n",
    "                metrics_acc[model_name].append(metrics['acc'])\n",
    "                counts[model_name].append(metrics['n'])\n",
    "            else:\n",
    "                metric_file = os.path.join(model_path, partition, epoch, dataset, 'metrics.npz')\n",
    "                if os.path.isfile(metric_file):\n",
    "                    # print(\"Using metrics.npz for %s\" % model_name)\n",
    "                    metrics = np.load(metric_file)\n",
    "                    metrics_ap[model_name].append(metrics['ap'])\n",
    "                    metrics_acc[model_name].append(metrics['acc'])\n",
    "                    counts[model_name].append(metrics['n'])\n",
    "                else:\n",
    "                    metrics_ap[model_name].append(np.nan)\n",
    "                    metrics_acc[model_name].append(np.nan)\n",
    "                    counts[model_name].append(0)\n",
    "    df_ap = pd.DataFrame(data=metrics_ap, index=dataset_names).T * 100\n",
    "    df_acc = pd.DataFrame(data=metrics_acc, index=dataset_names).T * 100\n",
    "    df_counts = pd.DataFrame(data=counts, index=dataset_names).T\n",
    "    # df_counts is for sanity checking the number of images processed\n",
    "    return df_ap, df_acc, df_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_listing_for_group(grp_string):\n",
    "    # grp_string = gp1 etc\n",
    "    expts = sorted(glob.glob('../results/%s-*block*' % grp_string))\n",
    "    names = ['-'.join(x.split('_')[-4:-2]) for x in expts]\n",
    "    baseline_expts = sorted(glob.glob('../results/%s-*baseline*' % grp_string))\n",
    "    baseline_names = ['-'.join(x.split('_')[-2:]) for x in baseline_expts]\n",
    "    # print(baseline_expts)\n",
    "    # print(baseline_names)\n",
    "    \n",
    "    if baseline_expts:\n",
    "        order = [4,0,3,5,1,2]\n",
    "        baseline_expts = [baseline_expts[i] for i in order]\n",
    "        baseline_names = [baseline_names[i] for i in order]\n",
    "    else:\n",
    "        print(\"No baselines found\")\n",
    "        \n",
    "    return expts+baseline_expts, names+baseline_names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  xception gen models tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "expts, names = get_listing_for_group('gp1')\n",
    "t1, t2, n = make_table(expts, names, dataset_names_list['gen_models'], 'avg_after_softmax', partition='test')\n",
    "display(t1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# random crop\n",
    "expts, names = get_listing_for_group('gp1b') # random crop\n",
    "t1, t2, n = make_table(expts, names, dataset_names_list['gen_models'], 'avg_after_softmax', partition='test')\n",
    "display(t1)\n",
    "\n",
    "# random resize crop\n",
    "expts, names = get_listing_for_group('gp1a') # random resize crop\n",
    "t1, t2, n = make_table(expts, names, dataset_names_list['gen_models'], 'avg_after_softmax', partition='test')\n",
    "display(t1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# samples only\n",
    "expts, names = get_listing_for_group('gp1d') # samples only\n",
    "t1, t2, n = make_table(expts, names, dataset_names_list['gen_models'], 'avg_after_softmax', partition='test')\n",
    "display(t1)\n",
    "\n",
    "# inverses only\n",
    "expts, names = get_listing_for_group('gp1c') # inverse only\n",
    "t1, t2, n = make_table(expts, names, dataset_names_list['gen_models'], 'avg_after_softmax', partition='test')\n",
    "display(t1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# face forensics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "expts, names = get_listing_for_group('gp5') # train on F2F\n",
    "t1, t2, n = make_table(expts, names, dataset_names_list['faceforensics'], 'avg_after_softmax', 'test')\n",
    "display(t1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "expts, names = get_listing_for_group('gp4') # train on FS\n",
    "t1, t2, n = make_table(expts, names, dataset_names_list['faceforensics'], 'avg_after_softmax', 'test')\n",
    "display(t1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "expts, names = get_listing_for_group('gp3') # train on NT\n",
    "t1, t2, n = make_table(expts, names, dataset_names_list['faceforensics'], 'avg_after_softmax', 'test')\n",
    "display(t1)\n",
    "table_nt = t1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "expts, names = get_listing_for_group('gp2') # train on DF\n",
    "t1, t2, n = make_table(expts, names, dataset_names_list['faceforensics'], 'avg_after_softmax', 'test')\n",
    "display(t1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "forgery",
   "language": "python",
   "name": "forgery"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}